{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:23:01.898998Z",
     "start_time": "2017-11-07T04:23:00.657322Z"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "import re\n",
    "import collections\n",
    "import itertools\n",
    "import bcolz\n",
    "import pickle\n",
    "sys.path.append('../../lib')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import random\n",
    "import smart_open\n",
    "import h5py\n",
    "import csv\n",
    "import json\n",
    "import functools\n",
    "import time\n",
    "import string\n",
    "\n",
    "import datetime as dt\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import global_utils\n",
    "\n",
    "random_state_number = 967898"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:23:02.379870Z",
     "start_time": "2017-11-07T04:23:01.900248Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/gpu:0', '/gpu:1']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.python.client import device_lib\n",
    "def get_available_gpus():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos if x.device_type == 'GPU']\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True\n",
    "sess = tf.Session(config=config)\n",
    "get_available_gpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:23:02.542383Z",
     "start_time": "2017-11-07T04:23:02.380926Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using matplotlib backend: Qt5Agg\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bicepjai/Programs/anaconda3/envs/dsotc-c3/lib/python3.6/site-packages/IPython/core/magics/pylab.py:160: UserWarning: pylab import has clobbered these variables: ['random']\n",
      "`%matplotlib` prevents importing * from pylab and numpy\n",
      "  \"\\n`%matplotlib` prevents importing * from pylab and numpy\"\n"
     ]
    }
   ],
   "source": [
    "%pylab\n",
    "%matplotlib inline\n",
    "%load_ext line_profiler\n",
    "%load_ext memory_profiler\n",
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:23:02.546392Z",
     "start_time": "2017-11-07T04:23:02.543671Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.options.mode.chained_assignment = None\n",
    "pd.options.display.max_columns = 999\n",
    "color = sns.color_palette()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:23:20.650863Z",
     "start_time": "2017-11-07T04:23:02.547357Z"
    }
   },
   "outputs": [],
   "source": [
    "store = pd.HDFStore('../../data_prep/processed/stage1/data_frames.h5')\n",
    "train_df = store['train_df']\n",
    "test_df = store['test_df']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:23:20.963869Z",
     "start_time": "2017-11-07T04:23:20.652089Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Gene</th>\n",
       "      <th>Variation</th>\n",
       "      <th>Class</th>\n",
       "      <th>Sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[fam58a]</td>\n",
       "      <td>[truncating, mutations]</td>\n",
       "      <td>1</td>\n",
       "      <td>[[cyclin-dependent, kinases, , cdks, , regulat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[w802*]</td>\n",
       "      <td>2</td>\n",
       "      <td>[[abstract, background, non-small, cell, lung,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[q249e]</td>\n",
       "      <td>2</td>\n",
       "      <td>[[abstract, background, non-small, cell, lung,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[n454d]</td>\n",
       "      <td>3</td>\n",
       "      <td>[[recent, evidence, has, demonstrated, that, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[l399v]</td>\n",
       "      <td>4</td>\n",
       "      <td>[[oncogenic, mutations, in, the, monomeric, ca...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID      Gene                Variation  Class  \\\n",
       "0   0  [fam58a]  [truncating, mutations]      1   \n",
       "1   1     [cbl]                  [w802*]      2   \n",
       "2   2     [cbl]                  [q249e]      2   \n",
       "3   3     [cbl]                  [n454d]      3   \n",
       "4   4     [cbl]                  [l399v]      4   \n",
       "\n",
       "                                           Sentences  \n",
       "0  [[cyclin-dependent, kinases, , cdks, , regulat...  \n",
       "1  [[abstract, background, non-small, cell, lung,...  \n",
       "2  [[abstract, background, non-small, cell, lung,...  \n",
       "3  [[recent, evidence, has, demonstrated, that, a...  \n",
       "4  [[oncogenic, mutations, in, the, monomeric, ca...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Gene</th>\n",
       "      <th>Variation</th>\n",
       "      <th>Sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[acsl4]</td>\n",
       "      <td>[r570s]</td>\n",
       "      <td>[[2, this, mutation, resulted, in, a, myelopro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[naglu]</td>\n",
       "      <td>[p521l]</td>\n",
       "      <td>[[abstract, the, large, tumor, suppressor, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[pah]</td>\n",
       "      <td>[l333f]</td>\n",
       "      <td>[[vascular, endothelial, growth, factor, recep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[ing1]</td>\n",
       "      <td>[a148d]</td>\n",
       "      <td>[[inflammatory, myofibroblastic, tumor, , imt,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[tmem216]</td>\n",
       "      <td>[g77a]</td>\n",
       "      <td>[[abstract, retinoblastoma, is, a, pediatric, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID       Gene Variation                                          Sentences\n",
       "0   0    [acsl4]   [r570s]  [[2, this, mutation, resulted, in, a, myelopro...\n",
       "1   1    [naglu]   [p521l]  [[abstract, the, large, tumor, suppressor, 1, ...\n",
       "2   2      [pah]   [l333f]  [[vascular, endothelial, growth, factor, recep...\n",
       "3   3     [ing1]   [a148d]  [[inflammatory, myofibroblastic, tumor, , imt,...\n",
       "4   4  [tmem216]    [g77a]  [[abstract, retinoblastoma, is, a, pediatric, ..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(train_df.head())\n",
    "display(test_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:23:21.066663Z",
     "start_time": "2017-11-07T04:23:20.965010Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "352220 352220\n"
     ]
    }
   ],
   "source": [
    "corpus_vocab_list, corpus_vocab_wordidx = None, None\n",
    "with open('../../data_prep/processed/stage1/vocab_words_wordidx.pkl', 'rb') as f:\n",
    "    (corpus_vocab_list, corpus_wordidx) = pickle.load(f)\n",
    "print(len(corpus_vocab_list), len(corpus_wordidx))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-14T08:20:17.449244Z",
     "start_time": "2017-08-14T08:20:15.593136Z"
    },
    "collapsed": true
   },
   "source": [
    "# Data Prep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To control the vocabulary pass in updated corpus_wordidx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:34.856451Z",
     "start_time": "2017-11-07T04:28:34.799259Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 5)\n",
      "(333, 5)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "x_train_df, x_val_df = train_test_split(train_df,\n",
    "                                         test_size=0.10, random_state=random_state_number,\n",
    "                                         stratify=train_df.Class)\n",
    "\n",
    "print(x_train_df.shape)\n",
    "print(x_val_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:35.048380Z",
     "start_time": "2017-11-07T04:28:35.042006Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7    858\n",
      "4    617\n",
      "1    511\n",
      "2    407\n",
      "6    247\n",
      "5    218\n",
      "3     80\n",
      "9     33\n",
      "8     17\n",
      "Name: Class, dtype: int64\n",
      "7    95\n",
      "4    69\n",
      "1    57\n",
      "2    45\n",
      "6    28\n",
      "5    24\n",
      "3     9\n",
      "9     4\n",
      "8     2\n",
      "Name: Class, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(x_train_df.Class.value_counts())\n",
    "print(x_val_df.Class.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:35.287941Z",
     "start_time": "2017-11-07T04:28:35.284599Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib.keras.python.keras.utils import np_utils\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.np_utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:36.375218Z",
     "start_time": "2017-11-07T04:28:36.370582Z"
    }
   },
   "outputs": [],
   "source": [
    "vocab_size=len(corpus_vocab_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## T:sent_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:25:05.352459Z",
     "start_time": "2017-09-25T06:25:05.349721Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"      : \"words\",\n",
    "         \"variation_unit\" : \"words\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"       : \"words\",\n",
    "         \"doc_form\"       : \"sentences\",\n",
    "         \"divide_document\": \"multiple_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:26:20.210322Z",
     "start_time": "2017-09-25T06:25:05.353617Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_21_T, x_train_21_G, x_train_21_V, x_train_21_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:26:21.998936Z",
     "start_time": "2017-09-25T06:26:20.211666Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "(1086419,) [352216, 252037, 202038, 70974, 86431, 164788, 109857, 338562, 123191, 209585, 221967, 49123, 331220, 140212, 209585, 229015, 140770, 182848, 111721, 8208, 0, 352217]\n",
      "(1086419, 3) [352216, 164788, 352217]\n",
      "(1086419,) [352216, 86196, 352217]\n",
      "(1086419,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(np.array(x_train_21_T).shape, x_train_21_T[0])\n",
    "print(np.array(x_train_21_G).shape, x_train_21_G[0])\n",
    "print(np.array(x_train_21_V).shape, x_train_21_V[0])\n",
    "print(np.array(x_train_21_C).shape, x_train_21_C[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:26:29.776970Z",
     "start_time": "2017-09-25T06:26:22.000201Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_21_T, x_val_21_G, x_val_21_V, x_val_21_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:26:29.999447Z",
     "start_time": "2017-09-25T06:26:29.778292Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val data\n",
      "text (128341,)\n",
      "gene (128341, 3) [352216, 217983, 352217]\n",
      "variation (128341,) [352216, 41934, 352217]\n",
      "classes (128341,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_21_T).shape)\n",
    "print(\"gene\",np.array(x_val_21_G).shape, x_val_21_G[0])\n",
    "print(\"variation\",np.array(x_val_21_V).shape, x_val_21_V[0])\n",
    "print(\"classes\",np.array(x_val_21_C).shape, x_val_21_C[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:26:30.003257Z",
     "start_time": "2017-09-25T06:26:30.000708Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:26:30.006725Z",
     "start_time": "2017-09-25T06:26:30.004479Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "MAX_SENT_LEN = 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:26:37.079748Z",
     "start_time": "2017-09-25T06:26:30.007947Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1086419, 60) (128341, 60)\n"
     ]
    }
   ],
   "source": [
    "x_train_21_T = pad_sequences(x_train_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "x_val_21_T = pad_sequences(x_val_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_train_21_T.shape, x_val_21_T.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:26:37.170342Z",
     "start_time": "2017-09-25T06:26:37.081558Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "x_train_21_C = np.array(x_train_21_C) - 1\n",
    "x_val_21_C = np.array(x_val_21_C) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:26:37.196926Z",
     "start_time": "2017-09-25T06:26:37.171834Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1086419, 9) (128341, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_21_C = np_utils.to_categorical(np.array(x_train_21_C), 9)\n",
    "x_val_21_C = np_utils.to_categorical(np.array(x_val_21_C), 9)\n",
    "print(x_train_21_C.shape, x_val_21_C.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## T:text_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:38.848466Z",
     "start_time": "2017-11-07T04:28:38.844619Z"
    }
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"      : \"words\",\n",
    "         \"variation_unit\" : \"words\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"       : \"words\",\n",
    "         \"doc_form\"       : \"text\",\n",
    "         \"divide_document\": \"single_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:49.319904Z",
     "start_time": "2017-11-07T04:28:39.119185Z"
    }
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_22_T, x_train_22_G, x_train_22_V, x_train_22_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:50.426965Z",
     "start_time": "2017-11-07T04:28:49.321123Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "text (2988,)\n",
      "gene (2988, 3) [352216, 164788, 352217]\n",
      "variation (2988,) [352216, 86196, 352217]\n",
      "classes (2988,) 4\n",
      "[1 2 3 4 5 6 7 8 9]\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(\"text\",np.array(x_train_22_T).shape)\n",
    "print(\"gene\",np.array(x_train_22_G).shape, x_train_22_G[0])\n",
    "print(\"variation\",np.array(x_train_22_V).shape, x_train_22_V[0])\n",
    "print(\"classes\",np.array(x_train_22_C).shape, x_train_22_C[0])\n",
    "print(unique(x_train_22_C))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:51.909055Z",
     "start_time": "2017-11-07T04:28:50.428145Z"
    }
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_22_T, x_val_22_G, x_val_22_V, x_val_22_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:52.047669Z",
     "start_time": "2017-11-07T04:28:51.910177Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val data\n",
      "text (333,)\n",
      "gene (333, 3) [352216, 217983, 352217]\n",
      "variation (333,) [352216, 41934, 352217]\n",
      "classes (333,) 4\n",
      "[1 2 3 4 5 6 7 8 9]\n"
     ]
    }
   ],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_22_T).shape)\n",
    "print(\"gene\",np.array(x_val_22_G).shape, x_val_22_G[0])\n",
    "print(\"variation\",np.array(x_val_22_V).shape, x_val_22_V[0])\n",
    "print(\"classes\",np.array(x_val_22_C).shape, x_val_22_C[0])\n",
    "print(unique(x_val_22_C))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:52.052729Z",
     "start_time": "2017-11-07T04:28:52.049256Z"
    }
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:52.057420Z",
     "start_time": "2017-11-07T04:28:52.054149Z"
    }
   },
   "outputs": [],
   "source": [
    "MAX_TEXT_LEN = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:53.193409Z",
     "start_time": "2017-11-07T04:28:52.058563Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 5000) (333, 5000)\n"
     ]
    }
   ],
   "source": [
    "x_train_22_T = pad_sequences(x_train_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "x_val_22_T = pad_sequences(x_val_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_train_22_T.shape, x_val_22_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:53.238060Z",
     "start_time": "2017-11-07T04:28:53.194569Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 1) (2988, 4)\n",
      "(333, 1) (333, 4)\n"
     ]
    }
   ],
   "source": [
    "MAX_GENE_LEN = 1\n",
    "MAX_VAR_LEN = 4\n",
    "x_train_22_G = pad_sequences(x_train_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_train_22_V = pad_sequences(x_train_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "x_val_22_G = pad_sequences(x_val_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_val_22_V = pad_sequences(x_val_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "print(x_train_22_G.shape, x_train_22_V.shape)\n",
    "print(x_val_22_G.shape, x_val_22_V.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:53.245708Z",
     "start_time": "2017-11-07T04:28:53.239272Z"
    }
   },
   "outputs": [],
   "source": [
    "x_train_22_C = np.array(x_train_22_C) - 1\n",
    "x_val_22_C = np.array(x_val_22_C) - 1\n",
    "x_train_22_Cp = x_train_22_C\n",
    "x_val_22_Cp = x_val_22_C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:53.251889Z",
     "start_time": "2017-11-07T04:28:53.246846Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 2 3 4 5 6 7 8]\n",
      "[0 1 2 3 4 5 6 7 8]\n"
     ]
    }
   ],
   "source": [
    "print(unique(x_val_22_Cp))\n",
    "print(unique(x_train_22_Cp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:53.257067Z",
     "start_time": "2017-11-07T04:28:53.253172Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 9) (333, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_22_C = np_utils.to_categorical(np.array(x_train_22_C), 9)\n",
    "x_val_22_C = np_utils.to_categorical(np.array(x_val_22_C), 9)\n",
    "print(x_train_22_C.shape, x_val_22_C.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:53.261836Z",
     "start_time": "2017-11-07T04:28:53.258154Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  1.]\n",
      "[ 0.  1.]\n"
     ]
    }
   ],
   "source": [
    "print(unique(x_train_22_C))\n",
    "print(unique(x_val_22_C))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CV setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:23:32.726283Z",
     "start_time": "2017-11-07T04:23:21.071774Z"
    }
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(train_df, corpus_wordidx)\n",
    "x_22_T, x_22_G, x_22_V, x_22_C = gen_data.generate_data(custom_unit_dict,\n",
    "                                                        has_class=True,\n",
    "                                                        add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:25:24.842796Z",
     "start_time": "2017-11-07T04:25:23.654056Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3321, 5000)\n"
     ]
    }
   ],
   "source": [
    "x_22_T = pad_sequences(x_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_22_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:25:24.846580Z",
     "start_time": "2017-11-07T04:25:24.843851Z"
    }
   },
   "outputs": [],
   "source": [
    "x_22_C = np.array(x_22_C) - 1\n",
    "x_22_Cp = x_22_C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:25:24.855164Z",
     "start_time": "2017-11-07T04:25:24.847722Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3321, 9)\n"
     ]
    }
   ],
   "source": [
    "x_22_C = np_utils.to_categorical(np.array(x_22_C), 9)\n",
    "print(x_22_C.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### test Data setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:25:44.190647Z",
     "start_time": "2017-11-07T04:25:25.794302Z"
    }
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(test_df, corpus_wordidx)\n",
    "x_test_22_T, x_test_22_G, x_test_22_V, _ = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                has_class=False,\n",
    "                                                                add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:25:46.090854Z",
     "start_time": "2017-11-07T04:25:44.191806Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test data\n",
      "text (5668,)\n",
      "gene (5668, 3) [352216, 136191, 352217]\n",
      "variation (5668,) [352216, 327792, 352217]\n"
     ]
    }
   ],
   "source": [
    "print(\"Test data\")\n",
    "print(\"text\",np.array(x_test_22_T).shape)\n",
    "print(\"gene\",np.array(x_test_22_G).shape, x_test_22_G[0])\n",
    "print(\"variation\",np.array(x_test_22_V).shape, x_test_22_V[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:25:48.069601Z",
     "start_time": "2017-11-07T04:25:46.091946Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5668, 5000)\n"
     ]
    }
   ],
   "source": [
    "x_test_22_T = pad_sequences(x_test_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_test_22_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:25:48.138294Z",
     "start_time": "2017-11-07T04:25:48.070916Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5668, 1) (5668, 4)\n"
     ]
    }
   ],
   "source": [
    "MAX_GENE_LEN = 1\n",
    "MAX_VAR_LEN = 4\n",
    "x_test_22_G = pad_sequences(x_test_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_test_22_V = pad_sequences(x_test_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "print(x_test_22_G.shape, x_test_22_V.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:25:48.158390Z",
     "start_time": "2017-11-07T04:25:48.140054Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>class1</th>\n",
       "      <th>class2</th>\n",
       "      <th>class3</th>\n",
       "      <th>class4</th>\n",
       "      <th>class5</th>\n",
       "      <th>class6</th>\n",
       "      <th>class7</th>\n",
       "      <th>class8</th>\n",
       "      <th>class9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  class1  class2  class3  class4  class5  class6  class7  class8  class9\n",
       "0   0       0       0       1       0       0       0       0       0       0\n",
       "1   1       0       0       0       0       0       0       0       0       1\n",
       "2   2       0       0       0       0       0       0       1       0       0\n",
       "3   3       0       1       0       0       0       0       0       0       0\n",
       "4   4       0       0       0       0       0       0       0       0       1"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1_solution_df = pd.read_csv(\"../../data_prep/dataset/stage2/stage1_solution.csv\")\n",
    "s1_solution_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:25:48.175595Z",
     "start_time": "2017-11-07T04:25:48.159655Z"
    }
   },
   "outputs": [],
   "source": [
    "test_matrix = s1_solution_df[['class1', 'class2', 'class3', 'class4', 'class5', 'class6', 'class7', 'class8', 'class9']]\n",
    "x_test_22_Cp = [np.where(r==1)[0][0] for r in test_matrix.values ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:25:48.181387Z",
     "start_time": "2017-11-07T04:25:48.176757Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5668, 9)\n"
     ]
    }
   ],
   "source": [
    "x_test_22_C = np.array(x_test_22_Cp) - 1\n",
    "x_test_22_C = np_utils.to_categorical(np.array(x_test_22_C), 9)\n",
    "print(x_test_22_C.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedding layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### for words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:25:48.184796Z",
     "start_time": "2017-11-07T04:25:48.182558Z"
    }
   },
   "outputs": [],
   "source": [
    "WORD_EMB_SIZE = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:26:11.234716Z",
     "start_time": "2017-11-07T04:25:48.185918Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(352220, 200)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "ft_file_path = \"/home/bicepjai/Projects/Deep-Survey-Text-Classification/data_prep/processed/stage1/pretrained_word_vectors/ft_sg_200d_50e.vec\"\n",
    "trained_embeddings = global_utils.get_embeddings_from_ft(ft_file_path, WORD_EMB_SIZE, corpus_vocab_list)\n",
    "trained_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### for characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:26:11.237544Z",
     "start_time": "2017-11-07T04:26:11.235777Z"
    }
   },
   "outputs": [],
   "source": [
    "CHAR_EMB_SIZE = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:26:11.246607Z",
     "start_time": "2017-11-07T04:26:11.238506Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75, 100)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_embeddings = np.random.randn(global_utils.CHAR_ALPHABETS_LEN, CHAR_EMB_SIZE)\n",
    "char_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:26:11.281792Z",
     "start_time": "2017-11-07T04:26:11.247791Z"
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow.contrib.keras as keras\n",
    "import tensorflow as tf\n",
    "\n",
    "from keras import backend as K\n",
    "\n",
    "from keras.engine import Layer, InputSpec, InputLayer\n",
    "\n",
    "from keras.models import Model, Sequential\n",
    "\n",
    "from keras.layers import Dropout, Embedding, concatenate\n",
    "from keras.layers import Conv1D, MaxPool1D, Conv2D, MaxPool2D, ZeroPadding1D\n",
    "from keras.layers import Dense, Input, Flatten, BatchNormalization\n",
    "from keras.layers import Concatenate, Dot, Merge, Multiply, RepeatVector\n",
    "from keras.layers import Bidirectional, TimeDistributed\n",
    "from keras.layers import SimpleRNN, LSTM, GRU, Lambda, Permute\n",
    "\n",
    "from keras.layers.core import Reshape, Activation\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint,EarlyStopping,TensorBoard\n",
    "from keras.constraints import maxnorm\n",
    "from keras.regularizers import l2\n",
    "\n",
    "%autoreload"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T06:58:17.661183Z",
     "start_time": "2017-08-24T06:58:17.655020Z"
    }
   },
   "source": [
    "## model_1: paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T03:33:20.907296Z",
     "start_time": "2017-11-07T03:33:20.245039Z"
    }
   },
   "outputs": [],
   "source": [
    "text_seq_input = Input(shape=(MAX_TEXT_LEN,), dtype='int32')\n",
    "text_embedding = Embedding(vocab_size, WORD_EMB_SIZE, input_length=MAX_TEXT_LEN,\n",
    "                            weights=[trained_embeddings], trainable=True)(text_seq_input)\n",
    "\n",
    "filter_sizes = [3,4,5]\n",
    "convs = []\n",
    "for filter_size in filter_sizes:\n",
    "    l_conv = Conv1D(filters=128, kernel_size=filter_size, padding='same', activation='relu')(text_embedding)\n",
    "    l_pool = MaxPool1D(filter_size)(l_conv)\n",
    "    convs.append(l_pool)\n",
    "\n",
    "l_merge = Concatenate(axis=1)(convs)\n",
    "l_cov1= Conv1D(128, 5, activation='relu')(l_merge)\n",
    "# since the text is too long we are maxpooling over 100\n",
    "# and not GlobalMaxPool1D\n",
    "l_pool1 = MaxPool1D(100)(l_cov1)\n",
    "l_flat = Flatten()(l_pool1)\n",
    "l_dense = Dense(128, activation='relu')(l_flat)\n",
    "l_out = Dense(9, activation='softmax')(l_dense)\n",
    "model_1 = Model(inputs=[text_seq_input], outputs=l_out)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T03:33:20.935035Z",
     "start_time": "2017-11-07T03:33:20.908462Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, 5000)          0                                            \n",
      "____________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)          (None, 5000, 200)     70444000    input_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)                (None, 5000, 128)     76928       embedding_1[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)                (None, 5000, 128)     102528      embedding_1[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)                (None, 5000, 128)     128128      embedding_1[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1D)   (None, 1666, 128)     0           conv1d_1[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1D)   (None, 1250, 128)     0           conv1d_2[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1D)   (None, 1000, 128)     0           conv1d_3[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 3916, 128)     0           max_pooling1d_1[0][0]            \n",
      "                                                                   max_pooling1d_2[0][0]            \n",
      "                                                                   max_pooling1d_3[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)                (None, 3912, 128)     82048       concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_4 (MaxPooling1D)   (None, 39, 128)       0           conv1d_4[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)              (None, 4992)          0           max_pooling1d_4[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 128)           639104      flatten_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 9)             1161        dense_1[0][0]                    \n",
      "====================================================================================================\n",
      "Total params: 71,473,897\n",
      "Trainable params: 71,473,897\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_1.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['categorical_accuracy'])\n",
    "model_1.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T03:37:04.282110Z",
     "start_time": "2017-11-07T03:37:04.277386Z"
    }
   },
   "outputs": [],
   "source": [
    "tb_callback = keras.callbacks.TensorBoard(log_dir='./tb_graphs', histogram_freq=0, write_graph=True, write_images=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T03:37:04.853479Z",
     "start_time": "2017-11-07T03:37:04.849453Z"
    }
   },
   "outputs": [],
   "source": [
    "checkpointer = ModelCheckpoint(filepath=\"model_1_weights.hdf5\", \n",
    "                                    verbose=1,\n",
    "                                    monitor=\"val_categorical_accuracy\",\n",
    "                                    save_best_only=True,\n",
    "                                    mode=\"max\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T03:37:05.162020Z",
     "start_time": "2017-11-07T03:37:05.158745Z"
    }
   },
   "outputs": [],
   "source": [
    "earlystopping = EarlyStopping(monitor='val_categorical_accuracy', \n",
    "                              min_delta=0, patience=5, \n",
    "                              verbose=0, mode='auto')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### no CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T03:13:56.094108Z",
     "start_time": "2017-11-07T03:12:07.772842Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no checkpoints available !\n",
      "Train on 2988 samples, validate on 333 samples\n",
      "Epoch 1/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 1.7705 - categorical_accuracy: 0.3346Epoch 00000: val_categorical_accuracy improved from -inf to 0.45045, saving model to model_1_weights.hdf5\n",
      "2988/2988 [==============================] - 10s - loss: 1.7673 - categorical_accuracy: 0.3377 - val_loss: 1.6119 - val_categorical_accuracy: 0.4505\n",
      "Epoch 2/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 1.3092 - categorical_accuracy: 0.5673Epoch 00001: val_categorical_accuracy improved from 0.45045 to 0.58859, saving model to model_1_weights.hdf5\n",
      "2988/2988 [==============================] - 11s - loss: 1.3063 - categorical_accuracy: 0.5683 - val_loss: 1.3341 - val_categorical_accuracy: 0.5886\n",
      "Epoch 3/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.9138 - categorical_accuracy: 0.7272Epoch 00002: val_categorical_accuracy improved from 0.58859 to 0.60961, saving model to model_1_weights.hdf5\n",
      "2988/2988 [==============================] - 11s - loss: 0.9135 - categorical_accuracy: 0.7269 - val_loss: 1.2460 - val_categorical_accuracy: 0.6096\n",
      "Epoch 4/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.7357 - categorical_accuracy: 0.7514Epoch 00003: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 9s - loss: 0.7339 - categorical_accuracy: 0.7517 - val_loss: 1.2656 - val_categorical_accuracy: 0.5946\n",
      "Epoch 5/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.6520 - categorical_accuracy: 0.7622Epoch 00004: val_categorical_accuracy improved from 0.60961 to 0.62763, saving model to model_1_weights.hdf5\n",
      "2988/2988 [==============================] - 11s - loss: 0.6537 - categorical_accuracy: 0.7610 - val_loss: 1.2111 - val_categorical_accuracy: 0.6276\n",
      "Epoch 6/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.6101 - categorical_accuracy: 0.7741Epoch 00005: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 9s - loss: 0.6103 - categorical_accuracy: 0.7741 - val_loss: 1.2082 - val_categorical_accuracy: 0.6066\n",
      "Epoch 7/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.5625 - categorical_accuracy: 0.7796Epoch 00006: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 9s - loss: 0.5627 - categorical_accuracy: 0.7798 - val_loss: 1.1969 - val_categorical_accuracy: 0.5886\n",
      "Epoch 8/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.5450 - categorical_accuracy: 0.7846Epoch 00007: val_categorical_accuracy improved from 0.62763 to 0.62763, saving model to model_1_weights.hdf5\n",
      "2988/2988 [==============================] - 11s - loss: 0.5466 - categorical_accuracy: 0.7845 - val_loss: 1.2106 - val_categorical_accuracy: 0.6276\n",
      "Epoch 9/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.5325 - categorical_accuracy: 0.7846Epoch 00008: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 9s - loss: 0.5313 - categorical_accuracy: 0.7855 - val_loss: 1.2050 - val_categorical_accuracy: 0.6036\n",
      "Epoch 10/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.5120 - categorical_accuracy: 0.7833Epoch 00009: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 9s - loss: 0.5108 - categorical_accuracy: 0.7841 - val_loss: 1.2568 - val_categorical_accuracy: 0.6066\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    try:\n",
    "        model_1.load_weights(\"model_11_weights.hdf5\")\n",
    "    except IOError as ioe:\n",
    "        print(\"no checkpoints available !\")\n",
    "    \n",
    "    model_1.fit(x_train_22_T, x_train_22_C, \n",
    "          validation_data=(x_val_22_T, x_val_22_C),\n",
    "          epochs=10, batch_size=64,shuffle=True,\n",
    "          callbacks=[tb_callback,checkpointer])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T03:14:52.800428Z",
     "start_time": "2017-11-07T03:14:47.627602Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 67  28   2  54  16   9 245   0   1]\n",
      " [ 85  71   2  87   8  13 473   0   2]\n",
      " [ 76  48   3  64   8   6 441   0   4]\n",
      " [ 79  57   1 101  16   9 447   0   1]\n",
      " [ 77  52   1  70  27  10 451   0   0]\n",
      " [ 90  56   2  65  14  21 454   0   6]\n",
      " [ 77  75   2  56  19   7 503   0   0]\n",
      " [ 72  47   3  72  11   6 456   0   2]\n",
      " [ 24  32   0  32  10   4 232   0   6]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.10      0.16      0.13       422\n",
      "          1       0.15      0.10      0.12       741\n",
      "          2       0.19      0.00      0.01       650\n",
      "          3       0.17      0.14      0.15       711\n",
      "          4       0.21      0.04      0.07       688\n",
      "          5       0.25      0.03      0.05       708\n",
      "          6       0.14      0.68      0.23       739\n",
      "          7       0.00      0.00      0.00       669\n",
      "          8       0.27      0.02      0.03       340\n",
      "\n",
      "avg / total       0.16      0.14      0.09      5668\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bicepjai/Programs/anaconda3/envs/dsotc-c3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    try:\n",
    "        model_1.load_weights(\"model_11_weights.hdf5\")\n",
    "    except IOError as ioe:\n",
    "        print(\"no checkpoints available !\")\n",
    "    \n",
    "    y_pred = model_1.predict(x_test_22_T)\n",
    "    y_classes = y_pred.argmax(axis=-1)\n",
    "\n",
    "print(confusion_matrix(x_test_22_Cp, y_classes))\n",
    "print(classification_report(x_test_22_Cp, y_classes))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T03:37:11.850736Z",
     "start_time": "2017-11-07T03:37:11.846710Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:04:33.606246Z",
     "start_time": "2017-11-07T04:04:33.602367Z"
    }
   },
   "outputs": [],
   "source": [
    "checkpointer = ModelCheckpoint(filepath=\"model_12_chk.hdf5\", \n",
    "                                    verbose=1,\n",
    "                                    monitor=\"val_categorical_accuracy\",\n",
    "                                    save_best_only=True,\n",
    "                                    mode=\"max\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:12:46.464990Z",
     "start_time": "2017-11-07T04:04:49.007880Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold 0 =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*\n",
      "no checkpoints available !\n",
      "Train on 2653 samples, validate on 668 samples\n",
      "Epoch 1/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 1.7498 - categorical_accuracy: 0.3453Epoch 00000: val_categorical_accuracy improved from -inf to 0.52545, saving model to model_12_chk.hdf5\n",
      "2653/2653 [==============================] - 11s - loss: 1.7488 - categorical_accuracy: 0.3464 - val_loss: 1.6020 - val_categorical_accuracy: 0.5254\n",
      "Epoch 2/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 1.2979 - categorical_accuracy: 0.5945Epoch 00001: val_categorical_accuracy improved from 0.52545 to 0.54790, saving model to model_12_chk.hdf5\n",
      "2653/2653 [==============================] - 10s - loss: 1.2945 - categorical_accuracy: 0.5956 - val_loss: 1.3016 - val_categorical_accuracy: 0.5479\n",
      "Epoch 3/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.8691 - categorical_accuracy: 0.7412Epoch 00002: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 9s - loss: 0.8735 - categorical_accuracy: 0.7395 - val_loss: 1.2846 - val_categorical_accuracy: 0.4970\n",
      "Epoch 4/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.7099 - categorical_accuracy: 0.7626Epoch 00003: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 9s - loss: 0.7130 - categorical_accuracy: 0.7614 - val_loss: 1.2404 - val_categorical_accuracy: 0.5434\n",
      "Epoch 5/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.6389 - categorical_accuracy: 0.7748Epoch 00004: val_categorical_accuracy improved from 0.54790 to 0.56886, saving model to model_12_chk.hdf5\n",
      "2653/2653 [==============================] - 10s - loss: 0.6434 - categorical_accuracy: 0.7731 - val_loss: 1.3267 - val_categorical_accuracy: 0.5689\n",
      "Epoch 6/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.6217 - categorical_accuracy: 0.7790Epoch 00005: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 9s - loss: 0.6245 - categorical_accuracy: 0.7776 - val_loss: 1.2580 - val_categorical_accuracy: 0.5479\n",
      "Epoch 7/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.5589 - categorical_accuracy: 0.7889Epoch 00006: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 9s - loss: 0.5592 - categorical_accuracy: 0.7874 - val_loss: 1.2553 - val_categorical_accuracy: 0.5659\n",
      "Epoch 8/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.5459 - categorical_accuracy: 0.7851Epoch 00007: val_categorical_accuracy improved from 0.56886 to 0.57036, saving model to model_12_chk.hdf5\n",
      "2653/2653 [==============================] - 11s - loss: 0.5463 - categorical_accuracy: 0.7851 - val_loss: 1.2700 - val_categorical_accuracy: 0.5704\n",
      "Epoch 9/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.5181 - categorical_accuracy: 0.7915Epoch 00008: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 9s - loss: 0.5194 - categorical_accuracy: 0.7912 - val_loss: 1.2886 - val_categorical_accuracy: 0.5359\n",
      "Epoch 10/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.5034 - categorical_accuracy: 0.7976Epoch 00009: val_categorical_accuracy improved from 0.57036 to 0.57485, saving model to model_12_chk.hdf5\n",
      "2653/2653 [==============================] - 10s - loss: 0.5020 - categorical_accuracy: 0.7980 - val_loss: 1.2252 - val_categorical_accuracy: 0.5749\n",
      "fold 1 =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*\n",
      "Train on 2654 samples, validate on 667 samples\n",
      "Epoch 1/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.6851 - categorical_accuracy: 0.7405Epoch 00000: val_categorical_accuracy improved from 0.57485 to 0.78561, saving model to model_12_chk.hdf5\n",
      "2654/2654 [==============================] - 11s - loss: 0.6853 - categorical_accuracy: 0.7408 - val_loss: 0.5595 - val_categorical_accuracy: 0.7856\n",
      "Epoch 2/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.5265 - categorical_accuracy: 0.7950Epoch 00001: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 9s - loss: 0.5299 - categorical_accuracy: 0.7935 - val_loss: 0.6224 - val_categorical_accuracy: 0.7511\n",
      "Epoch 3/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.4930 - categorical_accuracy: 0.7995Epoch 00002: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 9s - loss: 0.4968 - categorical_accuracy: 0.7977 - val_loss: 0.6817 - val_categorical_accuracy: 0.7496\n",
      "Epoch 4/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.4862 - categorical_accuracy: 0.7931Epoch 00003: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 9s - loss: 0.4863 - categorical_accuracy: 0.7935 - val_loss: 0.7085 - val_categorical_accuracy: 0.7511\n",
      "Epoch 5/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.4636 - categorical_accuracy: 0.8095Epoch 00004: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 9s - loss: 0.4645 - categorical_accuracy: 0.8097 - val_loss: 0.7526 - val_categorical_accuracy: 0.7556\n",
      "Epoch 6/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.4448 - categorical_accuracy: 0.8018Epoch 00005: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 9s - loss: 0.4436 - categorical_accuracy: 0.8026 - val_loss: 0.7705 - val_categorical_accuracy: 0.7616\n",
      "Epoch 7/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.4425 - categorical_accuracy: 0.8079Epoch 00006: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 9s - loss: 0.4410 - categorical_accuracy: 0.8090 - val_loss: 0.7882 - val_categorical_accuracy: 0.7496\n",
      "Epoch 8/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.4312 - categorical_accuracy: 0.8072Epoch 00007: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 9s - loss: 0.4311 - categorical_accuracy: 0.8078 - val_loss: 0.7901 - val_categorical_accuracy: 0.7646\n",
      "Epoch 9/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.4216 - categorical_accuracy: 0.8095Epoch 00008: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 9s - loss: 0.4239 - categorical_accuracy: 0.8082 - val_loss: 0.8149 - val_categorical_accuracy: 0.7631\n",
      "Epoch 10/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.4052 - categorical_accuracy: 0.8148Epoch 00009: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 9s - loss: 0.4104 - categorical_accuracy: 0.8131 - val_loss: 0.8808 - val_categorical_accuracy: 0.7361\n",
      "fold 2 =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*\n",
      "Train on 2657 samples, validate on 664 samples\n",
      "Epoch 1/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.5289 - categorical_accuracy: 0.7919Epoch 00000: val_categorical_accuracy improved from 0.78561 to 0.80271, saving model to model_12_chk.hdf5\n",
      "2657/2657 [==============================] - 10s - loss: 0.5288 - categorical_accuracy: 0.7919 - val_loss: 0.4491 - val_categorical_accuracy: 0.8027\n",
      "Epoch 2/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4494 - categorical_accuracy: 0.8011Epoch 00001: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 9s - loss: 0.4493 - categorical_accuracy: 0.8017 - val_loss: 0.4931 - val_categorical_accuracy: 0.7892\n",
      "Epoch 3/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4281 - categorical_accuracy: 0.8011Epoch 00002: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 9s - loss: 0.4278 - categorical_accuracy: 0.8009 - val_loss: 0.5374 - val_categorical_accuracy: 0.7696\n",
      "Epoch 4/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4234 - categorical_accuracy: 0.8121Epoch 00003: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 9s - loss: 0.4242 - categorical_accuracy: 0.8114 - val_loss: 0.6054 - val_categorical_accuracy: 0.7741\n",
      "Epoch 5/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4132 - categorical_accuracy: 0.8037Epoch 00004: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 9s - loss: 0.4132 - categorical_accuracy: 0.8035 - val_loss: 0.6082 - val_categorical_accuracy: 0.7500\n",
      "Epoch 6/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4010 - categorical_accuracy: 0.8152Epoch 00005: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 9s - loss: 0.4010 - categorical_accuracy: 0.8148 - val_loss: 0.6385 - val_categorical_accuracy: 0.7786\n",
      "Epoch 7/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4009 - categorical_accuracy: 0.8129Epoch 00006: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 9s - loss: 0.4009 - categorical_accuracy: 0.8133 - val_loss: 0.6459 - val_categorical_accuracy: 0.7681\n",
      "Epoch 8/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.3928 - categorical_accuracy: 0.8091Epoch 00007: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 9s - loss: 0.3949 - categorical_accuracy: 0.8096 - val_loss: 0.7110 - val_categorical_accuracy: 0.7696\n",
      "Epoch 9/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.3875 - categorical_accuracy: 0.8095Epoch 00008: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 9s - loss: 0.3863 - categorical_accuracy: 0.8103 - val_loss: 0.6911 - val_categorical_accuracy: 0.7711\n",
      "Epoch 10/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.3828 - categorical_accuracy: 0.8148Epoch 00009: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 9s - loss: 0.3812 - categorical_accuracy: 0.8160 - val_loss: 0.7392 - val_categorical_accuracy: 0.7741\n",
      "fold 3 =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*\n",
      "Train on 2659 samples, validate on 662 samples\n",
      "Epoch 1/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.4776 - categorical_accuracy: 0.8079Epoch 00000: val_categorical_accuracy improved from 0.80271 to 0.81269, saving model to model_12_chk.hdf5\n",
      "2659/2659 [==============================] - 10s - loss: 0.4764 - categorical_accuracy: 0.8074 - val_loss: 0.4080 - val_categorical_accuracy: 0.8127\n",
      "Epoch 2/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.4081 - categorical_accuracy: 0.8213Epoch 00001: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 9s - loss: 0.4068 - categorical_accuracy: 0.8225 - val_loss: 0.4649 - val_categorical_accuracy: 0.7915\n",
      "Epoch 3/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3904 - categorical_accuracy: 0.8186Epoch 00002: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 9s - loss: 0.3909 - categorical_accuracy: 0.8184 - val_loss: 0.5519 - val_categorical_accuracy: 0.7855\n",
      "Epoch 4/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3860 - categorical_accuracy: 0.8117Epoch 00003: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 9s - loss: 0.3855 - categorical_accuracy: 0.8120 - val_loss: 0.5712 - val_categorical_accuracy: 0.7810\n",
      "Epoch 5/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3764 - categorical_accuracy: 0.8197Epoch 00004: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 9s - loss: 0.3773 - categorical_accuracy: 0.8195 - val_loss: 0.6462 - val_categorical_accuracy: 0.7795\n",
      "Epoch 6/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3752 - categorical_accuracy: 0.8247Epoch 00005: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 9s - loss: 0.3746 - categorical_accuracy: 0.8247 - val_loss: 0.6767 - val_categorical_accuracy: 0.7840\n",
      "Epoch 7/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3736 - categorical_accuracy: 0.8190Epoch 00006: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 9s - loss: 0.3730 - categorical_accuracy: 0.8191 - val_loss: 0.7115 - val_categorical_accuracy: 0.7825\n",
      "Epoch 8/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3706 - categorical_accuracy: 0.8228Epoch 00007: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 9s - loss: 0.3699 - categorical_accuracy: 0.8225 - val_loss: 0.7410 - val_categorical_accuracy: 0.7915\n",
      "Epoch 9/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3690 - categorical_accuracy: 0.8277Epoch 00008: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 9s - loss: 0.3685 - categorical_accuracy: 0.8274 - val_loss: 0.7296 - val_categorical_accuracy: 0.7870\n",
      "Epoch 10/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3586 - categorical_accuracy: 0.8274Epoch 00009: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 9s - loss: 0.3604 - categorical_accuracy: 0.8263 - val_loss: 0.7143 - val_categorical_accuracy: 0.7810\n",
      "fold 4 =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*\n",
      "Train on 2661 samples, validate on 660 samples\n",
      "Epoch 1/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.4543 - categorical_accuracy: 0.8098Epoch 00000: val_categorical_accuracy improved from 0.81269 to 0.81970, saving model to model_12_chk.hdf5\n",
      "2661/2661 [==============================] - 10s - loss: 0.4511 - categorical_accuracy: 0.8113 - val_loss: 0.3918 - val_categorical_accuracy: 0.8197\n",
      "Epoch 2/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.3834 - categorical_accuracy: 0.8186Epoch 00001: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 9s - loss: 0.3843 - categorical_accuracy: 0.8185 - val_loss: 0.4175 - val_categorical_accuracy: 0.8045\n",
      "Epoch 3/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.3778 - categorical_accuracy: 0.8262Epoch 00002: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 9s - loss: 0.3792 - categorical_accuracy: 0.8253 - val_loss: 0.4932 - val_categorical_accuracy: 0.8015\n",
      "Epoch 4/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.3756 - categorical_accuracy: 0.8209Epoch 00003: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 9s - loss: 0.3760 - categorical_accuracy: 0.8204 - val_loss: 0.5261 - val_categorical_accuracy: 0.7970\n",
      "Epoch 5/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.3666 - categorical_accuracy: 0.8175Epoch 00004: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 9s - loss: 0.3662 - categorical_accuracy: 0.8177 - val_loss: 0.5771 - val_categorical_accuracy: 0.7894\n",
      "Epoch 6/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.3629 - categorical_accuracy: 0.8255Epoch 00005: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 9s - loss: 0.3626 - categorical_accuracy: 0.8260 - val_loss: 0.6609 - val_categorical_accuracy: 0.7924\n",
      "Epoch 7/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.3646 - categorical_accuracy: 0.8186Epoch 00006: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 9s - loss: 0.3623 - categorical_accuracy: 0.8207 - val_loss: 0.6431 - val_categorical_accuracy: 0.7833\n",
      "Epoch 8/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.3620 - categorical_accuracy: 0.8232Epoch 00007: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 9s - loss: 0.3608 - categorical_accuracy: 0.8238 - val_loss: 0.6936 - val_categorical_accuracy: 0.7788\n",
      "Epoch 9/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.3571 - categorical_accuracy: 0.8258Epoch 00008: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 9s - loss: 0.3594 - categorical_accuracy: 0.8249 - val_loss: 0.7796 - val_categorical_accuracy: 0.7909\n",
      "Epoch 10/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.3593 - categorical_accuracy: 0.8277Epoch 00009: val_categorical_accuracy did not improve\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2661/2661 [==============================] - 9s - loss: 0.3613 - categorical_accuracy: 0.8264 - val_loss: 0.7249 - val_categorical_accuracy: 0.7879\n"
     ]
    }
   ],
   "source": [
    "cv_kfold = StratifiedKFold(n_splits=5, shuffle=True)\n",
    "\n",
    "model_12 = None\n",
    "model_acc = 0\n",
    "for index, (train_indices, val_indices) in enumerate(cv_kfold.split(x_22_T, x_22_Cp)):\n",
    "    \n",
    "    print(\"fold\",index,\"=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*\")\n",
    "    xtrain, xval = x_22_T[train_indices], x_22_T[val_indices]\n",
    "    ytrain, yval = x_22_Cp[train_indices], x_22_Cp[val_indices]\n",
    "    ytrain = np_utils.to_categorical(np.array(ytrain), 9)\n",
    "    yval = np_utils.to_categorical(np.array(yval), 9)\n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        try:\n",
    "            model_1.load_weights(\"model_12_weights.hdf5\")\n",
    "        except IOError as ioe:\n",
    "            print(\"no checkpoints available !\")\n",
    "        \n",
    "        model_1.fit(xtrain, ytrain, \n",
    "              validation_data=(xval, yval),\n",
    "              epochs=10, batch_size=64,shuffle=True,\n",
    "              callbacks=[tb_callback,checkpointer])\n",
    "    \n",
    "        loss, acc = model_1.evaluate(xval, yval, verbose=0)\n",
    "        if model_acc < acc:\n",
    "            model_12 = model_1\n",
    "            model_12.save_weights(\"model_12_weights.hdf5\")\n",
    "            model_acc = acc\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T01:51:00.622994Z",
     "start_time": "2017-11-07T01:51:00.619822Z"
    }
   },
   "source": [
    "#### predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:14:31.338294Z",
     "start_time": "2017-11-07T04:14:26.298845Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[155  63   2  45   8  10 135   1   3]\n",
      " [222 134   4  71   7  19 278   2   4]\n",
      " [194 122   4  54   4  12 253   3   4]\n",
      " [202 132   1  91   9  17 254   1   4]\n",
      " [186 116   2  69  20  15 276   2   2]\n",
      " [220 123   3  55   6  28 261   2  10]\n",
      " [193 134   8  55  18   7 315   5   4]\n",
      " [194 110   5  56  11   6 275   4   8]\n",
      " [ 77  59   1  27   4   8 155   1   8]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.09      0.37      0.15       422\n",
      "          1       0.13      0.18      0.15       741\n",
      "          2       0.13      0.01      0.01       650\n",
      "          3       0.17      0.13      0.15       711\n",
      "          4       0.23      0.03      0.05       688\n",
      "          5       0.23      0.04      0.07       708\n",
      "          6       0.14      0.43      0.21       739\n",
      "          7       0.19      0.01      0.01       669\n",
      "          8       0.17      0.02      0.04       340\n",
      "\n",
      "avg / total       0.17      0.13      0.10      5668\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    try:\n",
    "        model_1.load_weights(\"model_12_weights.hdf5\")\n",
    "    except IOError as ioe:\n",
    "        print(\"no checkpoints available !\")\n",
    "    \n",
    "    y_pred = model_1.predict(x_test_22_T)\n",
    "    y_classes = y_pred.argmax(axis=-1)\n",
    "\n",
    "print(confusion_matrix(x_test_22_Cp, y_classes))\n",
    "print(classification_report(x_test_22_Cp, y_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T06:58:17.661183Z",
     "start_time": "2017-08-24T06:58:17.655020Z"
    }
   },
   "source": [
    "## model_2: refined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:26:23.411780Z",
     "start_time": "2017-11-07T04:26:22.642308Z"
    }
   },
   "outputs": [],
   "source": [
    "text_seq_input = Input(shape=(MAX_TEXT_LEN,), dtype='int32')\n",
    "text_embedding = Embedding(vocab_size, WORD_EMB_SIZE, input_length=MAX_TEXT_LEN,\n",
    "                            weights=[trained_embeddings], trainable=True)(text_seq_input)\n",
    "\n",
    "filter_sizes = [3, 4, 5, 10, 30, 50]\n",
    "convs = []\n",
    "for filter_size in filter_sizes:\n",
    "    l_conv = Conv1D(filters=128, kernel_size=filter_size, padding='same', activation='relu')(text_embedding)\n",
    "    l_pool = MaxPool1D(filter_size)(l_conv)\n",
    "    convs.append(l_pool)\n",
    "\n",
    "l_merge = Concatenate(axis=1)(convs)\n",
    "l_cov1= Conv1D(128, 5, activation='relu', kernel_regularizer= l2(0.01))(l_merge)\n",
    "l_pool1 = MaxPool1D(128)(l_cov1)\n",
    "\n",
    "l_flat = Flatten()(l_pool1)\n",
    "l_dense = Dense(128, activation='relu')(l_flat)\n",
    "l_out = Dense(9, activation='softmax')(l_dense)\n",
    "model_2 = Model(inputs=[text_seq_input], outputs=l_out)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:26:27.154196Z",
     "start_time": "2017-11-07T04:26:27.090531Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_2 (InputLayer)             (None, 5000)          0                                            \n",
      "____________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)          (None, 5000, 200)     70444000    input_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)                (None, 5000, 128)     76928       embedding_1[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)                (None, 5000, 128)     102528      embedding_1[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)                (None, 5000, 128)     128128      embedding_1[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)                (None, 5000, 128)     256128      embedding_1[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_5 (Conv1D)                (None, 5000, 128)     768128      embedding_1[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_6 (Conv1D)                (None, 5000, 128)     1280128     embedding_1[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1D)   (None, 1666, 128)     0           conv1d_1[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1D)   (None, 1250, 128)     0           conv1d_2[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1D)   (None, 1000, 128)     0           conv1d_3[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_4 (MaxPooling1D)   (None, 500, 128)      0           conv1d_4[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_5 (MaxPooling1D)   (None, 166, 128)      0           conv1d_5[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_6 (MaxPooling1D)   (None, 100, 128)      0           conv1d_6[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 4682, 128)     0           max_pooling1d_1[0][0]            \n",
      "                                                                   max_pooling1d_2[0][0]            \n",
      "                                                                   max_pooling1d_3[0][0]            \n",
      "                                                                   max_pooling1d_4[0][0]            \n",
      "                                                                   max_pooling1d_5[0][0]            \n",
      "                                                                   max_pooling1d_6[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "conv1d_7 (Conv1D)                (None, 4678, 128)     82048       concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "max_pooling1d_7 (MaxPooling1D)   (None, 36, 128)       0           conv1d_7[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)              (None, 4608)          0           max_pooling1d_7[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 128)           589952      flatten_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 9)             1161        dense_1[0][0]                    \n",
      "====================================================================================================\n",
      "Total params: 73,729,129\n",
      "Trainable params: 73,729,129\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_2.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['categorical_accuracy'])\n",
    "model_2.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:26:31.044229Z",
     "start_time": "2017-11-07T04:26:31.039298Z"
    }
   },
   "outputs": [],
   "source": [
    "tb_callback = keras.callbacks.TensorBoard(log_dir='./tb_graphs', histogram_freq=0, write_graph=True, write_images=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:28:02.231553Z",
     "start_time": "2017-11-07T04:28:02.226897Z"
    }
   },
   "outputs": [],
   "source": [
    "checkpointer = ModelCheckpoint(filepath=\"model_21_weights.hdf5\", \n",
    "                                    verbose=1,\n",
    "                                    monitor=\"val_categorical_accuracy\",\n",
    "                                    save_best_only=True,\n",
    "                                    mode=\"max\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:26:33.596497Z",
     "start_time": "2017-11-07T04:26:33.591708Z"
    }
   },
   "outputs": [],
   "source": [
    "earlystopping = EarlyStopping(monitor='val_categorical_accuracy', \n",
    "                              min_delta=0, patience=5, \n",
    "                              verbose=1, mode='auto')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### no CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:33:41.349085Z",
     "start_time": "2017-11-07T04:28:54.581471Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no checkpoints available !\n",
      "Train on 2988 samples, validate on 333 samples\n",
      "Epoch 1/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 2.4043 - categorical_accuracy: 0.3685Epoch 00000: val_categorical_accuracy improved from -inf to 0.44144, saving model to model_21_weights.hdf5\n",
      "2988/2988 [==============================] - 31s - loss: 2.3980 - categorical_accuracy: 0.3691 - val_loss: 1.8602 - val_categorical_accuracy: 0.4414\n",
      "Epoch 2/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 1.5442 - categorical_accuracy: 0.5187Epoch 00001: val_categorical_accuracy improved from 0.44144 to 0.59159, saving model to model_21_weights.hdf5\n",
      "2988/2988 [==============================] - 29s - loss: 1.5407 - categorical_accuracy: 0.5194 - val_loss: 1.3750 - val_categorical_accuracy: 0.5916\n",
      "Epoch 3/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 1.1770 - categorical_accuracy: 0.6311Epoch 00002: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 27s - loss: 1.1761 - categorical_accuracy: 0.6315 - val_loss: 1.2786 - val_categorical_accuracy: 0.5766\n",
      "Epoch 4/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.9931 - categorical_accuracy: 0.6967Epoch 00003: val_categorical_accuracy improved from 0.59159 to 0.63664, saving model to model_21_weights.hdf5\n",
      "2988/2988 [==============================] - 29s - loss: 0.9922 - categorical_accuracy: 0.6971 - val_loss: 1.1795 - val_categorical_accuracy: 0.6366\n",
      "Epoch 5/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.8791 - categorical_accuracy: 0.7405Epoch 00004: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 27s - loss: 0.8765 - categorical_accuracy: 0.7420 - val_loss: 1.2330 - val_categorical_accuracy: 0.6066\n",
      "Epoch 6/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.8097 - categorical_accuracy: 0.7595Epoch 00005: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 27s - loss: 0.8120 - categorical_accuracy: 0.7584 - val_loss: 1.1753 - val_categorical_accuracy: 0.6306\n",
      "Epoch 7/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.7477 - categorical_accuracy: 0.7629Epoch 00006: val_categorical_accuracy improved from 0.63664 to 0.64264, saving model to model_21_weights.hdf5\n",
      "2988/2988 [==============================] - 29s - loss: 0.7531 - categorical_accuracy: 0.7614 - val_loss: 1.1824 - val_categorical_accuracy: 0.6426\n",
      "Epoch 8/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.7089 - categorical_accuracy: 0.7673Epoch 00007: val_categorical_accuracy improved from 0.64264 to 0.64865, saving model to model_21_weights.hdf5\n",
      "2988/2988 [==============================] - 29s - loss: 0.7110 - categorical_accuracy: 0.7657 - val_loss: 1.1630 - val_categorical_accuracy: 0.6486\n",
      "Epoch 9/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.6771 - categorical_accuracy: 0.7711Epoch 00008: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 27s - loss: 0.6765 - categorical_accuracy: 0.7707 - val_loss: 1.1500 - val_categorical_accuracy: 0.6126\n",
      "Epoch 10/10\n",
      "2944/2988 [============================>.] - ETA: 0s - loss: 0.6502 - categorical_accuracy: 0.7711Epoch 00009: val_categorical_accuracy did not improve\n",
      "2988/2988 [==============================] - 27s - loss: 0.6489 - categorical_accuracy: 0.7714 - val_loss: 1.1513 - val_categorical_accuracy: 0.6096\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    try:\n",
    "        model_2.load_weights(\"model_21_weights.hdf5\")\n",
    "    except IOError as ioe:\n",
    "        print(\"no checkpoints available !\")\n",
    "    \n",
    "    model_2.fit(x_train_22_T, x_train_22_C, \n",
    "          validation_data=(x_val_22_T, x_val_22_C),\n",
    "          epochs=10, batch_size=64,shuffle=True,\n",
    "          callbacks=[tb_callback,checkpointer])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:37:55.451853Z",
     "start_time": "2017-11-07T04:37:45.747849Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[173  35   1  55  30   6 122   0   0]\n",
      " [226  91   1  76  41  11 295   0   0]\n",
      " [214  69   4  61  38   8 256   0   0]\n",
      " [218  81   0  97  49   6 260   0   0]\n",
      " [216  67   1  67  62   8 267   0   0]\n",
      " [246  76   1  62  51  21 251   0   0]\n",
      " [222 105   4  59  39   3 307   0   0]\n",
      " [208  77   3  61  41   9 269   0   1]\n",
      " [ 87  37   0  37  25   5 147   0   2]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.10      0.41      0.16       422\n",
      "          1       0.14      0.12      0.13       741\n",
      "          2       0.27      0.01      0.01       650\n",
      "          3       0.17      0.14      0.15       711\n",
      "          4       0.16      0.09      0.12       688\n",
      "          5       0.27      0.03      0.05       708\n",
      "          6       0.14      0.42      0.21       739\n",
      "          7       0.00      0.00      0.00       669\n",
      "          8       0.67      0.01      0.01       340\n",
      "\n",
      "avg / total       0.19      0.13      0.10      5668\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bicepjai/Programs/anaconda3/envs/dsotc-c3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    try:\n",
    "        model_2.load_weights(\"model_21_weights.hdf5\")\n",
    "    except IOError as ioe:\n",
    "        print(\"no checkpoints available !\")\n",
    "    \n",
    "    y_pred = model_2.predict(x_test_22_T)\n",
    "    y_classes = y_pred.argmax(axis=-1)\n",
    "\n",
    "print(confusion_matrix(x_test_22_Cp, y_classes))\n",
    "print(classification_report(x_test_22_Cp, y_classes))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:38:00.736648Z",
     "start_time": "2017-11-07T04:38:00.732462Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T04:38:01.752762Z",
     "start_time": "2017-11-07T04:38:01.747414Z"
    }
   },
   "outputs": [],
   "source": [
    "checkpointer = ModelCheckpoint(filepath=\"model_22_chk.hdf5\", \n",
    "                                    verbose=1,\n",
    "                                    monitor=\"val_categorical_accuracy\",\n",
    "                                    save_best_only=True,\n",
    "                                    mode=\"max\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T05:04:27.907239Z",
     "start_time": "2017-11-07T04:43:24.874500Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold 0 =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*\n",
      "no checkpoints available !\n",
      "Train on 2653 samples, validate on 668 samples\n",
      "Epoch 1/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 2.4730 - categorical_accuracy: 0.3445Epoch 00000: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 24s - loss: 2.4671 - categorical_accuracy: 0.3449 - val_loss: 1.9488 - val_categorical_accuracy: 0.3338\n",
      "Epoch 2/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 1.6751 - categorical_accuracy: 0.4840Epoch 00001: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 24s - loss: 1.6717 - categorical_accuracy: 0.4847 - val_loss: 1.5826 - val_categorical_accuracy: 0.4970\n",
      "Epoch 3/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 1.2800 - categorical_accuracy: 0.6044Epoch 00002: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 24s - loss: 1.2792 - categorical_accuracy: 0.6057 - val_loss: 1.3578 - val_categorical_accuracy: 0.5689\n",
      "Epoch 4/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 1.0418 - categorical_accuracy: 0.6875Epoch 00003: val_categorical_accuracy improved from 0.60629 to 0.60778, saving model to model_22_chk.hdf5\n",
      "2653/2653 [==============================] - 26s - loss: 1.0446 - categorical_accuracy: 0.6871 - val_loss: 1.3145 - val_categorical_accuracy: 0.6078\n",
      "Epoch 5/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.8911 - categorical_accuracy: 0.7416Epoch 00004: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 24s - loss: 0.8912 - categorical_accuracy: 0.7410 - val_loss: 1.3518 - val_categorical_accuracy: 0.5778\n",
      "Epoch 6/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.8344 - categorical_accuracy: 0.7485Epoch 00005: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 24s - loss: 0.8373 - categorical_accuracy: 0.7475 - val_loss: 1.2809 - val_categorical_accuracy: 0.5853\n",
      "Epoch 7/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.7899 - categorical_accuracy: 0.7595Epoch 00006: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 24s - loss: 0.7893 - categorical_accuracy: 0.7591 - val_loss: 1.3969 - val_categorical_accuracy: 0.5898\n",
      "Epoch 8/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.7258 - categorical_accuracy: 0.7633Epoch 00007: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 24s - loss: 0.7273 - categorical_accuracy: 0.7622 - val_loss: 1.2452 - val_categorical_accuracy: 0.6078\n",
      "Epoch 9/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.6835 - categorical_accuracy: 0.7744Epoch 00008: val_categorical_accuracy did not improve\n",
      "2653/2653 [==============================] - 24s - loss: 0.6810 - categorical_accuracy: 0.7746 - val_loss: 1.3094 - val_categorical_accuracy: 0.5958\n",
      "Epoch 10/10\n",
      "2624/2653 [============================>.] - ETA: 0s - loss: 0.6618 - categorical_accuracy: 0.7725Epoch 00009: val_categorical_accuracy improved from 0.60778 to 0.61976, saving model to model_22_chk.hdf5\n",
      "2653/2653 [==============================] - 26s - loss: 0.6643 - categorical_accuracy: 0.7708 - val_loss: 1.2167 - val_categorical_accuracy: 0.6198\n",
      "fold 1 =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*\n",
      "Train on 2654 samples, validate on 667 samples\n",
      "Epoch 1/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.7994 - categorical_accuracy: 0.7283Epoch 00000: val_categorical_accuracy improved from 0.61976 to 0.77511, saving model to model_22_chk.hdf5\n",
      "2654/2654 [==============================] - 28s - loss: 0.7984 - categorical_accuracy: 0.7287 - val_loss: 0.6808 - val_categorical_accuracy: 0.7751\n",
      "Epoch 2/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.6539 - categorical_accuracy: 0.7691Epoch 00001: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 24s - loss: 0.6528 - categorical_accuracy: 0.7698 - val_loss: 0.6930 - val_categorical_accuracy: 0.7646\n",
      "Epoch 3/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.6004 - categorical_accuracy: 0.7839Epoch 00002: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 24s - loss: 0.6016 - categorical_accuracy: 0.7841 - val_loss: 0.7073 - val_categorical_accuracy: 0.7436\n",
      "Epoch 4/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.5859 - categorical_accuracy: 0.7900Epoch 00003: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 24s - loss: 0.5838 - categorical_accuracy: 0.7909 - val_loss: 0.7475 - val_categorical_accuracy: 0.7676\n",
      "Epoch 5/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.5574 - categorical_accuracy: 0.7934Epoch 00004: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 24s - loss: 0.5581 - categorical_accuracy: 0.7924 - val_loss: 0.7455 - val_categorical_accuracy: 0.7646\n",
      "Epoch 6/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.5325 - categorical_accuracy: 0.7946Epoch 00005: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 24s - loss: 0.5337 - categorical_accuracy: 0.7943 - val_loss: 0.7550 - val_categorical_accuracy: 0.7451\n",
      "Epoch 7/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.5303 - categorical_accuracy: 0.7942Epoch 00006: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 24s - loss: 0.5310 - categorical_accuracy: 0.7946 - val_loss: 0.7947 - val_categorical_accuracy: 0.7526\n",
      "Epoch 8/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.5115 - categorical_accuracy: 0.7992Epoch 00007: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 24s - loss: 0.5144 - categorical_accuracy: 0.7969 - val_loss: 0.8074 - val_categorical_accuracy: 0.7541\n",
      "Epoch 9/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.4916 - categorical_accuracy: 0.8007Epoch 00008: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 24s - loss: 0.4903 - categorical_accuracy: 0.8011 - val_loss: 0.8387 - val_categorical_accuracy: 0.7391\n",
      "Epoch 10/10\n",
      "2624/2654 [============================>.] - ETA: 0s - loss: 0.4934 - categorical_accuracy: 0.7988Epoch 00009: val_categorical_accuracy did not improve\n",
      "2654/2654 [==============================] - 24s - loss: 0.4910 - categorical_accuracy: 0.7995 - val_loss: 0.8857 - val_categorical_accuracy: 0.7376\n",
      "fold 2 =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*\n",
      "Train on 2657 samples, validate on 664 samples\n",
      "Epoch 1/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.5986 - categorical_accuracy: 0.7843Epoch 00000: val_categorical_accuracy improved from 0.77511 to 0.79217, saving model to model_22_chk.hdf5\n",
      "2657/2657 [==============================] - 28s - loss: 0.5988 - categorical_accuracy: 0.7836 - val_loss: 0.5248 - val_categorical_accuracy: 0.7922\n",
      "Epoch 2/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.5126 - categorical_accuracy: 0.8022Epoch 00001: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 24s - loss: 0.5156 - categorical_accuracy: 0.8009 - val_loss: 0.5648 - val_categorical_accuracy: 0.7756\n",
      "Epoch 3/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4944 - categorical_accuracy: 0.8102Epoch 00002: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 24s - loss: 0.4952 - categorical_accuracy: 0.8092 - val_loss: 0.5844 - val_categorical_accuracy: 0.7651\n",
      "Epoch 4/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4750 - categorical_accuracy: 0.8045Epoch 00003: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 24s - loss: 0.4754 - categorical_accuracy: 0.8043 - val_loss: 0.5905 - val_categorical_accuracy: 0.7726\n",
      "Epoch 5/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4652 - categorical_accuracy: 0.8129Epoch 00004: val_categorical_accuracy did not improve\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2657/2657 [==============================] - 24s - loss: 0.4656 - categorical_accuracy: 0.8126 - val_loss: 0.6405 - val_categorical_accuracy: 0.7801\n",
      "Epoch 6/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4513 - categorical_accuracy: 0.8087Epoch 00005: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 24s - loss: 0.4507 - categorical_accuracy: 0.8092 - val_loss: 0.6611 - val_categorical_accuracy: 0.7666\n",
      "Epoch 7/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4423 - categorical_accuracy: 0.8152Epoch 00006: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 24s - loss: 0.4461 - categorical_accuracy: 0.8141 - val_loss: 0.6571 - val_categorical_accuracy: 0.7861\n",
      "Epoch 8/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4404 - categorical_accuracy: 0.8095Epoch 00007: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 24s - loss: 0.4405 - categorical_accuracy: 0.8092 - val_loss: 0.6686 - val_categorical_accuracy: 0.7741\n",
      "Epoch 9/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4320 - categorical_accuracy: 0.8102Epoch 00008: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 24s - loss: 0.4356 - categorical_accuracy: 0.8084 - val_loss: 0.6883 - val_categorical_accuracy: 0.7711\n",
      "Epoch 10/10\n",
      "2624/2657 [============================>.] - ETA: 0s - loss: 0.4355 - categorical_accuracy: 0.8129Epoch 00009: val_categorical_accuracy did not improve\n",
      "2657/2657 [==============================] - 24s - loss: 0.4325 - categorical_accuracy: 0.8145 - val_loss: 0.7109 - val_categorical_accuracy: 0.7636\n",
      "fold 3 =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*\n",
      "Train on 2659 samples, validate on 662 samples\n",
      "Epoch 1/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.5042 - categorical_accuracy: 0.8018Epoch 00000: val_categorical_accuracy improved from 0.79217 to 0.79909, saving model to model_22_chk.hdf5\n",
      "2659/2659 [==============================] - 28s - loss: 0.5087 - categorical_accuracy: 0.7999 - val_loss: 0.4921 - val_categorical_accuracy: 0.7991\n",
      "Epoch 2/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.4419 - categorical_accuracy: 0.8148Epoch 00001: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 24s - loss: 0.4423 - categorical_accuracy: 0.8142 - val_loss: 0.5427 - val_categorical_accuracy: 0.7764\n",
      "Epoch 3/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.4264 - categorical_accuracy: 0.8129Epoch 00002: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 24s - loss: 0.4237 - categorical_accuracy: 0.8146 - val_loss: 0.5743 - val_categorical_accuracy: 0.7704\n",
      "Epoch 4/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.4149 - categorical_accuracy: 0.8209Epoch 00003: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 24s - loss: 0.4119 - categorical_accuracy: 0.8225 - val_loss: 0.6180 - val_categorical_accuracy: 0.7447\n",
      "Epoch 5/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.4014 - categorical_accuracy: 0.8133Epoch 00004: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 24s - loss: 0.4024 - categorical_accuracy: 0.8127 - val_loss: 0.6729 - val_categorical_accuracy: 0.7568\n",
      "Epoch 6/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3952 - categorical_accuracy: 0.8224Epoch 00005: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 24s - loss: 0.3960 - categorical_accuracy: 0.8214 - val_loss: 0.6611 - val_categorical_accuracy: 0.7462\n",
      "Epoch 7/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3939 - categorical_accuracy: 0.8201Epoch 00006: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 24s - loss: 0.3951 - categorical_accuracy: 0.8195 - val_loss: 0.6758 - val_categorical_accuracy: 0.7477\n",
      "Epoch 8/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3960 - categorical_accuracy: 0.8209Epoch 00007: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 24s - loss: 0.3963 - categorical_accuracy: 0.8210 - val_loss: 0.7850 - val_categorical_accuracy: 0.7447\n",
      "Epoch 9/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3918 - categorical_accuracy: 0.8182Epoch 00008: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 24s - loss: 0.3918 - categorical_accuracy: 0.8184 - val_loss: 0.7055 - val_categorical_accuracy: 0.7492\n",
      "Epoch 10/10\n",
      "2624/2659 [============================>.] - ETA: 0s - loss: 0.3837 - categorical_accuracy: 0.8296Epoch 00009: val_categorical_accuracy did not improve\n",
      "2659/2659 [==============================] - 24s - loss: 0.3846 - categorical_accuracy: 0.8285 - val_loss: 0.7014 - val_categorical_accuracy: 0.7432\n",
      "fold 4 =*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*\n",
      "Train on 2661 samples, validate on 660 samples\n",
      "Epoch 1/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.5093 - categorical_accuracy: 0.7980Epoch 00000: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 26s - loss: 0.5074 - categorical_accuracy: 0.7989 - val_loss: 0.4850 - val_categorical_accuracy: 0.7788\n",
      "Epoch 2/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.4418 - categorical_accuracy: 0.8072Epoch 00001: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 24s - loss: 0.4412 - categorical_accuracy: 0.8076 - val_loss: 0.5466 - val_categorical_accuracy: 0.7758\n",
      "Epoch 3/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.4302 - categorical_accuracy: 0.8171Epoch 00002: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 24s - loss: 0.4303 - categorical_accuracy: 0.8162 - val_loss: 0.6032 - val_categorical_accuracy: 0.7712\n",
      "Epoch 4/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.4173 - categorical_accuracy: 0.8129Epoch 00003: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 24s - loss: 0.4158 - categorical_accuracy: 0.8136 - val_loss: 0.6016 - val_categorical_accuracy: 0.7500\n",
      "Epoch 5/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.4055 - categorical_accuracy: 0.8190Epoch 00004: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 24s - loss: 0.4073 - categorical_accuracy: 0.8189 - val_loss: 0.6425 - val_categorical_accuracy: 0.7470\n",
      "Epoch 6/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.4049 - categorical_accuracy: 0.8205Epoch 00005: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 24s - loss: 0.4019 - categorical_accuracy: 0.8219 - val_loss: 0.6947 - val_categorical_accuracy: 0.7576\n",
      "Epoch 7/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.4021 - categorical_accuracy: 0.8266Epoch 00006: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 24s - loss: 0.4013 - categorical_accuracy: 0.8264 - val_loss: 0.7055 - val_categorical_accuracy: 0.7530\n",
      "Epoch 8/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.3949 - categorical_accuracy: 0.8213Epoch 00007: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 24s - loss: 0.3951 - categorical_accuracy: 0.8211 - val_loss: 0.7124 - val_categorical_accuracy: 0.7545\n",
      "Epoch 9/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.3938 - categorical_accuracy: 0.8224Epoch 00008: val_categorical_accuracy did not improve\n",
      "2661/2661 [==============================] - 24s - loss: 0.3927 - categorical_accuracy: 0.8215 - val_loss: 0.7809 - val_categorical_accuracy: 0.7576\n",
      "Epoch 10/10\n",
      "2624/2661 [============================>.] - ETA: 0s - loss: 0.3863 - categorical_accuracy: 0.8232Epoch 00009: val_categorical_accuracy did not improve\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2661/2661 [==============================] - 24s - loss: 0.3877 - categorical_accuracy: 0.8226 - val_loss: 0.7656 - val_categorical_accuracy: 0.7606\n"
     ]
    }
   ],
   "source": [
    "cv_kfold = StratifiedKFold(n_splits=5, shuffle=True)\n",
    "\n",
    "model_22 = None\n",
    "model_acc = 0\n",
    "for index, (train_indices, val_indices) in enumerate(cv_kfold.split(x_22_T, x_22_Cp)):\n",
    "    \n",
    "    print(\"fold\",index,\"=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*\")\n",
    "    xtrain, xval = x_22_T[train_indices], x_22_T[val_indices]\n",
    "    ytrain, yval = x_22_Cp[train_indices], x_22_Cp[val_indices]\n",
    "    ytrain = np_utils.to_categorical(np.array(ytrain), 9)\n",
    "    yval = np_utils.to_categorical(np.array(yval), 9)\n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        try:\n",
    "            model_2.load_weights(\"model_22_weights.hdf5\")\n",
    "        except IOError as ioe:\n",
    "            print(\"no checkpoints available !\")\n",
    "        \n",
    "        model_2.fit(xtrain, ytrain, \n",
    "              validation_data=(xval, yval),\n",
    "              epochs=10, batch_size=64,shuffle=True,\n",
    "              callbacks=[tb_callback,checkpointer])\n",
    "    \n",
    "        loss, acc = model_2.evaluate(xval, yval, verbose=0)\n",
    "        if model_acc < acc:\n",
    "            model_22 = model_2\n",
    "            model_22.save_weights(\"model_22_weights.hdf5\")\n",
    "            model_acc = acc\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T01:51:00.622994Z",
     "start_time": "2017-11-07T01:51:00.619822Z"
    }
   },
   "source": [
    "#### predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-07T05:04:37.647775Z",
     "start_time": "2017-11-07T05:04:27.909589Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[148  46   2  86   5   8 123   1   3]\n",
      " [174 110   3 134   6  12 296   2   4]\n",
      " [153  86   5 117   5   6 269   2   7]\n",
      " [165  85   1 154   6  11 283   3   3]\n",
      " [174  93   3 125  21  10 258   3   1]\n",
      " [188  88   5 125   7  23 261   2   9]\n",
      " [170  99   7 116  16   4 321   3   3]\n",
      " [164 102   3 107  10   8 263   5   7]\n",
      " [ 70  41   1  59   6   7 146   2   8]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.11      0.35      0.16       422\n",
      "          1       0.15      0.15      0.15       741\n",
      "          2       0.17      0.01      0.01       650\n",
      "          3       0.15      0.22      0.18       711\n",
      "          4       0.26      0.03      0.05       688\n",
      "          5       0.26      0.03      0.06       708\n",
      "          6       0.14      0.43      0.22       739\n",
      "          7       0.22      0.01      0.01       669\n",
      "          8       0.18      0.02      0.04       340\n",
      "\n",
      "avg / total       0.18      0.14      0.10      5668\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    try:\n",
    "        model_2.load_weights(\"model_22_weights.hdf5\")\n",
    "    except IOError as ioe:\n",
    "        print(\"no checkpoints available !\")\n",
    "    \n",
    "    y_pred = model_2.predict(x_test_22_T)\n",
    "    y_classes = y_pred.argmax(axis=-1)\n",
    "\n",
    "print(confusion_matrix(x_test_22_Cp, y_classes))\n",
    "print(classification_report(x_test_22_Cp, y_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {
    "height": "827px",
    "left": "0px",
    "right": "1192px",
    "top": "52px",
    "width": "300px"
   },
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
